{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "save_path = \"/data/zyang/datasets/amazon_book_new/\"\n",
    "train_ = pd.read_pickle(save_path+\"train_ood2.pkl\")\n",
    "valid_ = pd.read_pickle(save_path+\"valid_ood2.pkl\")\n",
    "test_ = pd.read_pickle(save_path+\"test_ood2.pkl\")\n",
    "data = pd.concat([train_,valid_,test_],axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((25747, 9), (25747, 9))"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid_.shape, test_.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uid</th>\n",
       "      <th>iid</th>\n",
       "      <th>label</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>his</th>\n",
       "      <th>his_title</th>\n",
       "      <th>title</th>\n",
       "      <th>flag</th>\n",
       "      <th>not_cold</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>1</td>\n",
       "      <td>6108</td>\n",
       "      <td>1</td>\n",
       "      <td>1.512259e+09</td>\n",
       "      <td>[0, 3054, 24657, 1798, 5404, 15904, 1344, 6688...</td>\n",
       "      <td>[, ENDURANCE: SHACKLETON'S INCREDIBLE VOYAGE b...</td>\n",
       "      <td>Summerlost</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>3</td>\n",
       "      <td>15867</td>\n",
       "      <td>0</td>\n",
       "      <td>1.512518e+09</td>\n",
       "      <td>[0]</td>\n",
       "      <td>[]</td>\n",
       "      <td>The Tempest</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    uid    iid  label     timestamp  \\\n",
       "50    1   6108      1  1.512259e+09   \n",
       "68    3  15867      0  1.512518e+09   \n",
       "\n",
       "                                                  his  \\\n",
       "50  [0, 3054, 24657, 1798, 5404, 15904, 1344, 6688...   \n",
       "68                                                [0]   \n",
       "\n",
       "                                            his_title        title  flag  \\\n",
       "50  [, ENDURANCE: SHACKLETON'S INCREDIBLE VOYAGE b...   Summerlost   0.0   \n",
       "68                                                 []  The Tempest   0.0   \n",
       "\n",
       "    not_cold  \n",
       "50         1  \n",
       "68         0  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid_.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uid</th>\n",
       "      <th>iid</th>\n",
       "      <th>label</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>his</th>\n",
       "      <th>his_title</th>\n",
       "      <th>title</th>\n",
       "      <th>flag</th>\n",
       "      <th>not_cold</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>1</td>\n",
       "      <td>872</td>\n",
       "      <td>0</td>\n",
       "      <td>1.513987e+09</td>\n",
       "      <td>[0, 3054, 24657, 1798, 5404, 15904, 1344, 6688...</td>\n",
       "      <td>[, ENDURANCE: SHACKLETON'S INCREDIBLE VOYAGE b...</td>\n",
       "      <td>God's Battalions: The Case for the Crusades</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>1</td>\n",
       "      <td>12784</td>\n",
       "      <td>0</td>\n",
       "      <td>1.514246e+09</td>\n",
       "      <td>[0, 3054, 24657, 1798, 5404, 15904, 1344, 6688...</td>\n",
       "      <td>[, ENDURANCE: SHACKLETON'S INCREDIBLE VOYAGE b...</td>\n",
       "      <td>The Royal Treatment: A Crown Jewels Romantic C...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    uid    iid  label     timestamp  \\\n",
       "51    1    872      0  1.513987e+09   \n",
       "52    1  12784      0  1.514246e+09   \n",
       "\n",
       "                                                  his  \\\n",
       "51  [0, 3054, 24657, 1798, 5404, 15904, 1344, 6688...   \n",
       "52  [0, 3054, 24657, 1798, 5404, 15904, 1344, 6688...   \n",
       "\n",
       "                                            his_title  \\\n",
       "51  [, ENDURANCE: SHACKLETON'S INCREDIBLE VOYAGE b...   \n",
       "52  [, ENDURANCE: SHACKLETON'S INCREDIBLE VOYAGE b...   \n",
       "\n",
       "                                                title  flag  not_cold  \n",
       "51        God's Battalions: The Case for the Crusades   1.0         1  \n",
       "52  The Royal Treatment: A Crown Jewels Romantic C...   1.0         1  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(778957, 9)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((727463, 9), (25747, 9), (25747, 9))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_.shape,valid_.shape, test_.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "label    8.0\n",
       "Name: 0.1, dtype: float64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_info = train_.groupby('uid').agg({'label':'count'})\n",
    "user_info.quantile(0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(22854, 1)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_info.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>uid</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22953</th>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22955</th>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22960</th>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22962</th>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22964</th>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>19525 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       label\n",
       "uid         \n",
       "1         50\n",
       "4         28\n",
       "6         41\n",
       "8         39\n",
       "9         31\n",
       "...      ...\n",
       "22953     22\n",
       "22955     16\n",
       "22960     18\n",
       "22962     11\n",
       "22964     28\n",
       "\n",
       "[19525 rows x 1 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_info[user_info['label']>10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test_[['uid','iid']].apply(lambda x: x.uid in train_user and x.iid in train_item, axis=1).astype(\"int\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>33713.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>21.578115</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>33.510363</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>12.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>24.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1060.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              label\n",
       "count  33713.000000\n",
       "mean      21.578115\n",
       "std       33.510363\n",
       "min        1.000000\n",
       "25%        5.000000\n",
       "50%       12.000000\n",
       "75%       24.000000\n",
       "max     1060.000000"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "item_info = train_.groupby('iid').agg({'label':'count'})\n",
    "item_info.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "label    3.0\n",
       "Name: 0.1, dtype: float64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "item_info.quantile(0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (2252409866.py, line 1)",
     "output_type": "error",
     "traceback": [
      "\u001B[0;36m  Cell \u001B[0;32mIn[11], line 1\u001B[0;36m\u001B[0m\n\u001B[0;31m    test_['warm'] =\u001B[0m\n\u001B[0m                    ^\u001B[0m\n\u001B[0;31mSyntaxError\u001B[0m\u001B[0;31m:\u001B[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "test_['warm'] = None"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "minigpt4",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
